{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "812b8417",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark \n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "#SparkSQL的许多功能封装在SparkSession的方法接口中\n",
    "\n",
    "spark = SparkSession.builder \\\n",
    "        .appName(\"test\") \\\n",
    "        .config(\"master\",\"local[4]\") \\\n",
    "        .enableHiveSupport() \\\n",
    "        .getOrCreate()\n",
    "\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ced60cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark.sql.functions as F\n",
    "from datetime import date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3fe647f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+---------+----+----------+------+\n",
      "|ssno|   ssname|ssex| sbirthday|sclass|\n",
      "+----+---------+----+----------+------+\n",
      "| 108|  qiudong|   m|1977-09-01| 95033|\n",
      "| 105|kuangming|   m|1975-10-02| 95031|\n",
      "| 107|   wangli|   w|1976-01-23| 95033|\n",
      "| 101|    lijun|   m|1975-02-20| 95031|\n",
      "| 109| wangfang|   w|1975-02-10| 95031|\n",
      "| 103|    lujun|   m|1974-06-03| 95031|\n",
      "+----+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Student = spark.createDataFrame(\n",
    "    data=[(108, 'qiudong', 'm', date(1977, 9, 1), 95033), (105, 'kuangming', 'm', date(1975, 10, 2), 95031),\n",
    "         (107, 'wangli', 'w', date(1976, 1, 23), 95033), (101, 'lijun', 'm', date(1975, 2, 20), 95031),\n",
    "         (109, 'wangfang', 'w', date(1975, 2, 10), 95031), (103, 'lujun', 'm', date(1974, 6, 3), 95031)],\n",
    "    schema=['ssno', 'ssname', 'ssex', 'sbirthday', 'sclass'])\n",
    "Student.show(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a6c7301a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------+----+\n",
      "| ccno|     cname|ctno|\n",
      "+-----+----------+----+\n",
      "|3-105|计算机导论| 825|\n",
      "|3-245|  操作系统| 804|\n",
      "|6-166|  数字电路| 856|\n",
      "|9-888|  高等数学| 831|\n",
      "+-----+----------+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Course = spark.createDataFrame(\n",
    "    data=[('3-105', '计算机导论', 825), ('3-245', '操作系统', 804), ('6-166', '数字电路', 856),\n",
    "         ('9-888', '高等数学', 831)],\n",
    "    schema=['ccno', 'cname', 'ctno'])\n",
    "Course.show(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "14fb65e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+------+\n",
      "|sno|  cno|degree|\n",
      "+---+-----+------+\n",
      "|103|3-245|    86|\n",
      "|105|3-245|    75|\n",
      "|109|3-245|    68|\n",
      "|103|3-105|    92|\n",
      "|105|3-105|    88|\n",
      "|109|3-105|    76|\n",
      "|101|3-105|    64|\n",
      "|107|3-105|    91|\n",
      "|108|3-105|    78|\n",
      "|101|6-166|    85|\n",
      "|107|6-166|    79|\n",
      "|108|6-166|    81|\n",
      "+---+-----+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Score = spark.createDataFrame(\n",
    "    data=[(103, '3-245', 86), (105, '3-245', 75), (109, '3-245', 68), (103, '3-105', 92), (105, '3-105', 88),\n",
    "         (109, '3-105', 76), (101, '3-105', 64), (107, '3-105', 91), (108, '3-105', 78), (101, '6-166', 85),\n",
    "         (107, '6-166', 79), (108, '6-166', 81)],\n",
    "    schema=['sno', 'cno', 'degree'])\n",
    "Score.show(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eca8ac21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+----+----------+----------+----------+\n",
      "|ttno|   tname|tsex| tbirthday|     tprof|   tdepart|\n",
      "+----+--------+----+----------+----------+----------+\n",
      "| 804| licheng|   m|1958-12-02|fujiaoshou|  计算机系|\n",
      "| 856| zhangxu|   m|1969-03-12|  jiangshi|电子工程系|\n",
      "| 825|wangping|   w|1972-05-05|   zhujiao|  计算机系|\n",
      "| 831| liubing|   w|1977-08-14|   zhujiao|电子工程系|\n",
      "+----+--------+----+----------+----------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Teacher = spark.createDataFrame(\n",
    "    data=[(804, 'licheng', 'm', date(1958, 12, 2), 'fujiaoshou', '计算机系'),\n",
    "         (856, 'zhangxu', 'm', date(1969, 3, 12), 'jiangshi', '电子工程系'),\n",
    "         (825, 'wangping', 'w', date(1972, 5, 5), 'zhujiao', '计算机系'),\n",
    "         (831, 'liubing', 'w', date(1977, 8, 14), 'zhujiao', '电子工程系')],\n",
    "    schema=['ttno', 'tname', 'tsex', 'tbirthday', 'tprof', 'tdepart'])\n",
    "Teacher.show(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7f0af1bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import Window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6aa1aa5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+----+----------+----------+----------+----------------------+\n",
      "|ttno|tname   |tsex|tbirthday |tprof     |tdepart   |shiyan                |\n",
      "+----+--------+----+----------+----------+----------+----------------------+\n",
      "|804 |licheng |m   |1958-12-02|fujiaoshou|计算机系  |[计算机系, 电子工程系]|\n",
      "|856 |zhangxu |m   |1969-03-12|jiangshi  |电子工程系|[计算机系, 电子工程系]|\n",
      "|825 |wangping|w   |1972-05-05|zhujiao   |计算机系  |[计算机系, 电子工程系]|\n",
      "|831 |liubing |w   |1977-08-14|zhujiao   |电子工程系|[计算机系, 电子工程系]|\n",
      "+----+--------+----+----------+----------+----------+----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Teacher1 = Teacher.withColumn('shiyan', F.collect_set(F.col('tdepart')).over(Window.partitionBy(F.col('tsex'))))\n",
    "Teacher1.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7fdcb38b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+----+----------+----------+----------+----------------------+-------+\n",
      "|ttno|tname   |tsex|tbirthday |tprof     |tdepart   |shiyan                |shiyan2|\n",
      "+----+--------+----+----------+----------+----------+----------------------+-------+\n",
      "|804 |licheng |m   |1958-12-02|fujiaoshou|计算机系  |[计算机系, 电子工程系]|2      |\n",
      "|856 |zhangxu |m   |1969-03-12|jiangshi  |电子工程系|[计算机系, 电子工程系]|2      |\n",
      "|825 |wangping|w   |1972-05-05|zhujiao   |计算机系  |[计算机系, 电子工程系]|2      |\n",
      "|831 |liubing |w   |1977-08-14|zhujiao   |电子工程系|[计算机系, 电子工程系]|2      |\n",
      "+----+--------+----+----------+----------+----------+----------------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Teacher2 = Teacher1.withColumn('shiyan2', F.size(F.col('shiyan')))\n",
    "Teacher2.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e44bf285",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+----+----------+----------+----------+-------+\n",
      "|ttno|tname   |tsex|tbirthday |tprof     |tdepart   |shiyan3|\n",
      "+----+--------+----+----------+----------+----------+-------+\n",
      "|804 |licheng |m   |1958-12-02|fujiaoshou|计算机系  |2      |\n",
      "|856 |zhangxu |m   |1969-03-12|jiangshi  |电子工程系|2      |\n",
      "|825 |wangping|w   |1972-05-05|zhujiao   |计算机系  |2      |\n",
      "|831 |liubing |w   |1977-08-14|zhujiao   |电子工程系|2      |\n",
      "+----+--------+----+----------+----------+----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Teacher3 = Teacher.withColumn('shiyan3', \\\n",
    "                              F.size(F.collect_set(F.col('tdepart')).over(Window.partitionBy(F.col('tsex')))))\n",
    "Teacher3.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b85afa2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+----+\n",
      "|t  |name|val |\n",
      "+---+----+----+\n",
      "|0  |long|0   |\n",
      "|1  |yun |100 |\n",
      "|2  |long|200 |\n",
      "|3  |yun |300 |\n",
      "|4  |yun |400 |\n",
      "|5  |long|500 |\n",
      "|6  |long|600 |\n",
      "|7  |xing|700 |\n",
      "|8  |song|800 |\n",
      "|9  |yun |900 |\n",
      "|10 |song|1000|\n",
      "|11 |song|1100|\n",
      "|12 |yun |1200|\n",
      "|13 |song|1300|\n",
      "|14 |song|1400|\n",
      "|15 |xing|1500|\n",
      "|16 |yun |1600|\n",
      "|17 |xing|1700|\n",
      "|18 |long|1800|\n",
      "|19 |long|1900|\n",
      "|20 |yun |2000|\n",
      "|21 |xing|2100|\n",
      "|22 |xing|2200|\n",
      "|23 |yun |2300|\n",
      "|24 |long|2400|\n",
      "|25 |yun |2500|\n",
      "|26 |long|2600|\n",
      "|27 |long|2700|\n",
      "|28 |yun |2800|\n",
      "|29 |song|2900|\n",
      "|30 |long|3000|\n",
      "|31 |long|3100|\n",
      "|32 |long|3200|\n",
      "|33 |xing|3300|\n",
      "|34 |yun |3400|\n",
      "|35 |yun |3500|\n",
      "|36 |song|3600|\n",
      "|37 |xing|3700|\n",
      "|38 |long|3800|\n",
      "|39 |song|3900|\n",
      "|40 |xing|4000|\n",
      "|41 |xing|4100|\n",
      "|42 |long|4200|\n",
      "|43 |long|4300|\n",
      "|44 |song|4400|\n",
      "|45 |xing|4500|\n",
      "|46 |yun |4600|\n",
      "|47 |xing|4700|\n",
      "|48 |song|4800|\n",
      "|49 |long|4900|\n",
      "|50 |yun |5000|\n",
      "|51 |long|5100|\n",
      "|52 |yun |5200|\n",
      "|53 |xing|5300|\n",
      "|54 |long|5400|\n",
      "|55 |xing|5500|\n",
      "|56 |song|5600|\n",
      "|57 |song|5700|\n",
      "|58 |song|5800|\n",
      "|59 |song|5900|\n",
      "|60 |song|6000|\n",
      "|61 |song|6100|\n",
      "|62 |yun |6200|\n",
      "|63 |long|6300|\n",
      "|64 |yun |6400|\n",
      "|65 |yun |6500|\n",
      "|66 |song|6600|\n",
      "|67 |yun |6700|\n",
      "|68 |xing|6800|\n",
      "|69 |long|6900|\n",
      "|70 |song|7000|\n",
      "|71 |song|7100|\n",
      "|72 |xing|7200|\n",
      "|73 |xing|7300|\n",
      "|74 |yun |7400|\n",
      "|75 |xing|7500|\n",
      "|76 |xing|7600|\n",
      "|77 |long|7700|\n",
      "|78 |xing|7800|\n",
      "|79 |song|7900|\n",
      "|80 |song|8000|\n",
      "|81 |song|8100|\n",
      "|82 |xing|8200|\n",
      "|83 |long|8300|\n",
      "|84 |yun |8400|\n",
      "|85 |xing|8500|\n",
      "|86 |long|8600|\n",
      "|87 |long|8700|\n",
      "|88 |xing|8800|\n",
      "|89 |yun |8900|\n",
      "|90 |yun |9000|\n",
      "|91 |song|9100|\n",
      "|92 |xing|9200|\n",
      "|93 |yun |9300|\n",
      "|94 |xing|9400|\n",
      "|95 |long|9500|\n",
      "|96 |song|9600|\n",
      "|97 |yun |9700|\n",
      "|98 |song|9800|\n",
      "|99 |xing|9900|\n",
      "+---+----+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#滑动窗口session特征、序列特征构造\n",
    "import numpy as np\n",
    "import random\n",
    "t = list(range(100))\n",
    "random.shuffle(t)\n",
    "name = ['song', 'yun', 'long', 'xing'] * 25\n",
    "value = [i*100 for i in t]\n",
    "# list(zip(t, name, value))\n",
    "from pyspark.sql.types import StructType, StructField, StringType, MapType, IntegerType, DoubleType\n",
    "schema = StructType([ \\\n",
    "    StructField('t',IntegerType(),True), \\\n",
    "    StructField('name',StringType(),True), \\\n",
    "    StructField('val',IntegerType(),True)\n",
    "  ])\n",
    "df_ex1 = spark.createDataFrame(\n",
    "    data=list(zip(t, name, value)), \\\n",
    "    schema=schema\n",
    ")\n",
    "import pyspark.sql.functions as F\n",
    "df_ex1 = df_ex1.orderBy(F.col('t').asc())\n",
    "df_ex1.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ee236d64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+---+--------+\n",
      "|t  |name|val|t_val   |\n",
      "+---+----+---+--------+\n",
      "|0  |xing|0  |{0, 0}  |\n",
      "|1  |song|1  |{1, 1}  |\n",
      "|2  |song|2  |{2, 2}  |\n",
      "|3  |long|3  |{3, 3}  |\n",
      "|4  |long|4  |{4, 4}  |\n",
      "|5  |long|5  |{5, 5}  |\n",
      "|6  |long|6  |{6, 6}  |\n",
      "|7  |long|7  |{7, 7}  |\n",
      "|8  |long|8  |{8, 8}  |\n",
      "|9  |song|9  |{9, 9}  |\n",
      "|10 |long|10 |{10, 10}|\n",
      "|11 |yun |11 |{11, 11}|\n",
      "|12 |long|12 |{12, 12}|\n",
      "|13 |xing|13 |{13, 13}|\n",
      "|14 |xing|14 |{14, 14}|\n",
      "|15 |long|15 |{15, 15}|\n",
      "|16 |xing|16 |{16, 16}|\n",
      "|17 |yun |17 |{17, 17}|\n",
      "|18 |yun |18 |{18, 18}|\n",
      "|19 |long|19 |{19, 19}|\n",
      "|20 |song|20 |{20, 20}|\n",
      "|21 |yun |21 |{21, 21}|\n",
      "|22 |long|22 |{22, 22}|\n",
      "|23 |long|23 |{23, 23}|\n",
      "|24 |yun |24 |{24, 24}|\n",
      "|25 |long|25 |{25, 25}|\n",
      "|26 |song|26 |{26, 26}|\n",
      "|27 |song|27 |{27, 27}|\n",
      "|28 |song|28 |{28, 28}|\n",
      "|29 |xing|29 |{29, 29}|\n",
      "|30 |yun |30 |{30, 30}|\n",
      "|31 |yun |31 |{31, 31}|\n",
      "|32 |yun |32 |{32, 32}|\n",
      "|33 |xing|33 |{33, 33}|\n",
      "|34 |yun |34 |{34, 34}|\n",
      "|35 |song|35 |{35, 35}|\n",
      "|36 |song|36 |{36, 36}|\n",
      "|37 |xing|37 |{37, 37}|\n",
      "|38 |song|38 |{38, 38}|\n",
      "|39 |yun |39 |{39, 39}|\n",
      "|40 |long|40 |{40, 40}|\n",
      "|41 |long|41 |{41, 41}|\n",
      "|42 |yun |42 |{42, 42}|\n",
      "|43 |xing|43 |{43, 43}|\n",
      "|44 |song|44 |{44, 44}|\n",
      "|45 |yun |45 |{45, 45}|\n",
      "|46 |yun |46 |{46, 46}|\n",
      "|47 |yun |47 |{47, 47}|\n",
      "|48 |song|48 |{48, 48}|\n",
      "|49 |long|49 |{49, 49}|\n",
      "|50 |yun |50 |{50, 50}|\n",
      "|51 |song|51 |{51, 51}|\n",
      "|52 |xing|52 |{52, 52}|\n",
      "|53 |yun |53 |{53, 53}|\n",
      "|54 |yun |54 |{54, 54}|\n",
      "|55 |xing|55 |{55, 55}|\n",
      "|56 |xing|56 |{56, 56}|\n",
      "|57 |yun |57 |{57, 57}|\n",
      "|58 |long|58 |{58, 58}|\n",
      "|59 |xing|59 |{59, 59}|\n",
      "|60 |song|60 |{60, 60}|\n",
      "|61 |yun |61 |{61, 61}|\n",
      "|62 |xing|62 |{62, 62}|\n",
      "|63 |song|63 |{63, 63}|\n",
      "|64 |song|64 |{64, 64}|\n",
      "|65 |xing|65 |{65, 65}|\n",
      "|66 |yun |66 |{66, 66}|\n",
      "|67 |yun |67 |{67, 67}|\n",
      "|68 |song|68 |{68, 68}|\n",
      "|69 |xing|69 |{69, 69}|\n",
      "|70 |long|70 |{70, 70}|\n",
      "|71 |long|71 |{71, 71}|\n",
      "|72 |long|72 |{72, 72}|\n",
      "|73 |yun |73 |{73, 73}|\n",
      "|74 |long|74 |{74, 74}|\n",
      "|75 |long|75 |{75, 75}|\n",
      "|76 |xing|76 |{76, 76}|\n",
      "|77 |song|77 |{77, 77}|\n",
      "|78 |yun |78 |{78, 78}|\n",
      "|79 |xing|79 |{79, 79}|\n",
      "|80 |xing|80 |{80, 80}|\n",
      "|81 |long|81 |{81, 81}|\n",
      "|82 |yun |82 |{82, 82}|\n",
      "|83 |yun |83 |{83, 83}|\n",
      "|84 |song|84 |{84, 84}|\n",
      "|85 |song|85 |{85, 85}|\n",
      "|86 |song|86 |{86, 86}|\n",
      "|87 |xing|87 |{87, 87}|\n",
      "|88 |long|88 |{88, 88}|\n",
      "|89 |xing|89 |{89, 89}|\n",
      "|90 |long|90 |{90, 90}|\n",
      "|91 |song|91 |{91, 91}|\n",
      "|92 |song|92 |{92, 92}|\n",
      "|93 |song|93 |{93, 93}|\n",
      "|94 |xing|94 |{94, 94}|\n",
      "|95 |xing|95 |{95, 95}|\n",
      "|96 |xing|96 |{96, 96}|\n",
      "|97 |xing|97 |{97, 97}|\n",
      "|98 |xing|98 |{98, 98}|\n",
      "|99 |song|99 |{99, 99}|\n",
      "+---+----+---+--------+\n",
      "\n",
      "root\n",
      " |-- t: long (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- val: long (nullable = true)\n",
      " |-- t_val: struct (nullable = true)\n",
      " |    |-- _1: long (nullable = true)\n",
      " |    |-- _2: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_ex1_temp = df_ex1.rdd.map(lambda x: (x[0], x[1], x[2], (x[0], x[2]))).toDF(['t', 'name', 'val', 't_val'])\n",
    "df_ex1_temp.show(100, False)\n",
    "df_ex1_temp.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "334d6e17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+----+----------+\n",
      "|t  |name|val |t_val     |\n",
      "+---+----+----+----------+\n",
      "|0  |long|0   |[0, 0]    |\n",
      "|1  |yun |100 |[1, 100]  |\n",
      "|2  |long|200 |[2, 200]  |\n",
      "|3  |yun |300 |[3, 300]  |\n",
      "|4  |yun |400 |[4, 400]  |\n",
      "|5  |long|500 |[5, 500]  |\n",
      "|6  |long|600 |[6, 600]  |\n",
      "|7  |xing|700 |[7, 700]  |\n",
      "|8  |song|800 |[8, 800]  |\n",
      "|9  |yun |900 |[9, 900]  |\n",
      "|10 |song|1000|[10, 1000]|\n",
      "|11 |song|1100|[11, 1100]|\n",
      "|12 |yun |1200|[12, 1200]|\n",
      "|13 |song|1300|[13, 1300]|\n",
      "|14 |song|1400|[14, 1400]|\n",
      "|15 |xing|1500|[15, 1500]|\n",
      "|16 |yun |1600|[16, 1600]|\n",
      "|17 |xing|1700|[17, 1700]|\n",
      "|18 |long|1800|[18, 1800]|\n",
      "|19 |long|1900|[19, 1900]|\n",
      "|20 |yun |2000|[20, 2000]|\n",
      "|21 |xing|2100|[21, 2100]|\n",
      "|22 |xing|2200|[22, 2200]|\n",
      "|23 |yun |2300|[23, 2300]|\n",
      "|24 |long|2400|[24, 2400]|\n",
      "|25 |yun |2500|[25, 2500]|\n",
      "|26 |long|2600|[26, 2600]|\n",
      "|27 |long|2700|[27, 2700]|\n",
      "|28 |yun |2800|[28, 2800]|\n",
      "|29 |song|2900|[29, 2900]|\n",
      "|30 |long|3000|[30, 3000]|\n",
      "|31 |long|3100|[31, 3100]|\n",
      "|32 |long|3200|[32, 3200]|\n",
      "|33 |xing|3300|[33, 3300]|\n",
      "|34 |yun |3400|[34, 3400]|\n",
      "|35 |yun |3500|[35, 3500]|\n",
      "|36 |song|3600|[36, 3600]|\n",
      "|37 |xing|3700|[37, 3700]|\n",
      "|38 |long|3800|[38, 3800]|\n",
      "|39 |song|3900|[39, 3900]|\n",
      "|40 |xing|4000|[40, 4000]|\n",
      "|41 |xing|4100|[41, 4100]|\n",
      "|42 |long|4200|[42, 4200]|\n",
      "|43 |long|4300|[43, 4300]|\n",
      "|44 |song|4400|[44, 4400]|\n",
      "|45 |xing|4500|[45, 4500]|\n",
      "|46 |yun |4600|[46, 4600]|\n",
      "|47 |xing|4700|[47, 4700]|\n",
      "|48 |song|4800|[48, 4800]|\n",
      "|49 |long|4900|[49, 4900]|\n",
      "|50 |yun |5000|[50, 5000]|\n",
      "|51 |long|5100|[51, 5100]|\n",
      "|52 |yun |5200|[52, 5200]|\n",
      "|53 |xing|5300|[53, 5300]|\n",
      "|54 |long|5400|[54, 5400]|\n",
      "|55 |xing|5500|[55, 5500]|\n",
      "|56 |song|5600|[56, 5600]|\n",
      "|57 |song|5700|[57, 5700]|\n",
      "|58 |song|5800|[58, 5800]|\n",
      "|59 |song|5900|[59, 5900]|\n",
      "|60 |song|6000|[60, 6000]|\n",
      "|61 |song|6100|[61, 6100]|\n",
      "|62 |yun |6200|[62, 6200]|\n",
      "|63 |long|6300|[63, 6300]|\n",
      "|64 |yun |6400|[64, 6400]|\n",
      "|65 |yun |6500|[65, 6500]|\n",
      "|66 |song|6600|[66, 6600]|\n",
      "|67 |yun |6700|[67, 6700]|\n",
      "|68 |xing|6800|[68, 6800]|\n",
      "|69 |long|6900|[69, 6900]|\n",
      "|70 |song|7000|[70, 7000]|\n",
      "|71 |song|7100|[71, 7100]|\n",
      "|72 |xing|7200|[72, 7200]|\n",
      "|73 |xing|7300|[73, 7300]|\n",
      "|74 |yun |7400|[74, 7400]|\n",
      "|75 |xing|7500|[75, 7500]|\n",
      "|76 |xing|7600|[76, 7600]|\n",
      "|77 |long|7700|[77, 7700]|\n",
      "|78 |xing|7800|[78, 7800]|\n",
      "|79 |song|7900|[79, 7900]|\n",
      "|80 |song|8000|[80, 8000]|\n",
      "|81 |song|8100|[81, 8100]|\n",
      "|82 |xing|8200|[82, 8200]|\n",
      "|83 |long|8300|[83, 8300]|\n",
      "|84 |yun |8400|[84, 8400]|\n",
      "|85 |xing|8500|[85, 8500]|\n",
      "|86 |long|8600|[86, 8600]|\n",
      "|87 |long|8700|[87, 8700]|\n",
      "|88 |xing|8800|[88, 8800]|\n",
      "|89 |yun |8900|[89, 8900]|\n",
      "|90 |yun |9000|[90, 9000]|\n",
      "|91 |song|9100|[91, 9100]|\n",
      "|92 |xing|9200|[92, 9200]|\n",
      "|93 |yun |9300|[93, 9300]|\n",
      "|94 |xing|9400|[94, 9400]|\n",
      "|95 |long|9500|[95, 9500]|\n",
      "|96 |song|9600|[96, 9600]|\n",
      "|97 |yun |9700|[97, 9700]|\n",
      "|98 |song|9800|[98, 9800]|\n",
      "|99 |xing|9900|[99, 9900]|\n",
      "+---+----+----+----------+\n",
      "\n",
      "root\n",
      " |-- t: integer (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- val: integer (nullable = true)\n",
      " |-- t_val: array (nullable = false)\n",
      " |    |-- element: integer (containsNull = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_ex1_temp2 = df_ex1.withColumn('t_val', F.array(F.col('t'), F.col('val')))\n",
    "df_ex1_temp2.show(100, False)\n",
    "df_ex1_temp2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "45ed8f82",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+----+----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|t  |name|val |t_val     |seq_t_val                                                                                                                                                                                                                                                                                                 |\n",
      "+---+----+----+----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|0  |long|0   |[0, 0]    |[[0, 0]]                                                                                                                                                                                                                                                                                                  |\n",
      "|2  |long|200 |[2, 200]  |[[0, 0], [2, 200]]                                                                                                                                                                                                                                                                                        |\n",
      "|5  |long|500 |[5, 500]  |[[0, 0], [2, 200], [5, 500]]                                                                                                                                                                                                                                                                              |\n",
      "|6  |long|600 |[6, 600]  |[[0, 0], [2, 200], [5, 500], [6, 600]]                                                                                                                                                                                                                                                                    |\n",
      "|18 |long|1800|[18, 1800]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800]]                                                                                                                                                                                                                                                        |\n",
      "|19 |long|1900|[19, 1900]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900]]                                                                                                                                                                                                                                            |\n",
      "|24 |long|2400|[24, 2400]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400]]                                                                                                                                                                                                                                |\n",
      "|26 |long|2600|[26, 2600]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600]]                                                                                                                                                                                                                    |\n",
      "|27 |long|2700|[27, 2700]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700]]                                                                                                                                                                                                        |\n",
      "|30 |long|3000|[30, 3000]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000]]                                                                                                                                                                                            |\n",
      "|31 |long|3100|[31, 3100]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100]]                                                                                                                                                                                |\n",
      "|32 |long|3200|[32, 3200]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200]]                                                                                                                                                                    |\n",
      "|38 |long|3800|[38, 3800]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800]]                                                                                                                                                        |\n",
      "|42 |long|4200|[42, 4200]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200]]                                                                                                                                            |\n",
      "|43 |long|4300|[43, 4300]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300]]                                                                                                                                |\n",
      "|49 |long|4900|[49, 4900]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900]]                                                                                                                    |\n",
      "|51 |long|5100|[51, 5100]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100]]                                                                                                        |\n",
      "|54 |long|5400|[54, 5400]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400]]                                                                                            |\n",
      "|63 |long|6300|[63, 6300]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300]]                                                                                |\n",
      "|69 |long|6900|[69, 6900]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900]]                                                                    |\n",
      "|77 |long|7700|[77, 7700]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700]]                                                        |\n",
      "|83 |long|8300|[83, 8300]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300]]                                            |\n",
      "|86 |long|8600|[86, 8600]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600]]                                |\n",
      "|87 |long|8700|[87, 8700]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600], [87, 8700]]                    |\n",
      "|95 |long|9500|[95, 9500]|[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600], [87, 8700], [95, 9500]]        |\n",
      "|8  |song|800 |[8, 800]  |[[8, 800]]                                                                                                                                                                                                                                                                                                |\n",
      "|10 |song|1000|[10, 1000]|[[8, 800], [10, 1000]]                                                                                                                                                                                                                                                                                    |\n",
      "|11 |song|1100|[11, 1100]|[[8, 800], [10, 1000], [11, 1100]]                                                                                                                                                                                                                                                                        |\n",
      "|13 |song|1300|[13, 1300]|[[8, 800], [10, 1000], [11, 1100], [13, 1300]]                                                                                                                                                                                                                                                            |\n",
      "|14 |song|1400|[14, 1400]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400]]                                                                                                                                                                                                                                                |\n",
      "|29 |song|2900|[29, 2900]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900]]                                                                                                                                                                                                                                    |\n",
      "|36 |song|3600|[36, 3600]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600]]                                                                                                                                                                                                                        |\n",
      "|39 |song|3900|[39, 3900]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900]]                                                                                                                                                                                                            |\n",
      "|44 |song|4400|[44, 4400]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400]]                                                                                                                                                                                                |\n",
      "|48 |song|4800|[48, 4800]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800]]                                                                                                                                                                                    |\n",
      "|56 |song|5600|[56, 5600]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600]]                                                                                                                                                                        |\n",
      "|57 |song|5700|[57, 5700]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700]]                                                                                                                                                            |\n",
      "|58 |song|5800|[58, 5800]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800]]                                                                                                                                                |\n",
      "|59 |song|5900|[59, 5900]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900]]                                                                                                                                    |\n",
      "|60 |song|6000|[60, 6000]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000]]                                                                                                                        |\n",
      "|61 |song|6100|[61, 6100]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100]]                                                                                                            |\n",
      "|66 |song|6600|[66, 6600]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600]]                                                                                                |\n",
      "|70 |song|7000|[70, 7000]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000]]                                                                                    |\n",
      "|71 |song|7100|[71, 7100]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100]]                                                                        |\n",
      "|79 |song|7900|[79, 7900]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900]]                                                            |\n",
      "|80 |song|8000|[80, 8000]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000]]                                                |\n",
      "|81 |song|8100|[81, 8100]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100]]                                    |\n",
      "|91 |song|9100|[91, 9100]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100]]                        |\n",
      "|96 |song|9600|[96, 9600]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100], [96, 9600]]            |\n",
      "|98 |song|9800|[98, 9800]|[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100], [96, 9600], [98, 9800]]|\n",
      "|7  |xing|700 |[7, 700]  |[[7, 700]]                                                                                                                                                                                                                                                                                                |\n",
      "|15 |xing|1500|[15, 1500]|[[7, 700], [15, 1500]]                                                                                                                                                                                                                                                                                    |\n",
      "|17 |xing|1700|[17, 1700]|[[7, 700], [15, 1500], [17, 1700]]                                                                                                                                                                                                                                                                        |\n",
      "|21 |xing|2100|[21, 2100]|[[7, 700], [15, 1500], [17, 1700], [21, 2100]]                                                                                                                                                                                                                                                            |\n",
      "|22 |xing|2200|[22, 2200]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200]]                                                                                                                                                                                                                                                |\n",
      "|33 |xing|3300|[33, 3300]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300]]                                                                                                                                                                                                                                    |\n",
      "|37 |xing|3700|[37, 3700]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700]]                                                                                                                                                                                                                        |\n",
      "|40 |xing|4000|[40, 4000]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000]]                                                                                                                                                                                                            |\n",
      "|41 |xing|4100|[41, 4100]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100]]                                                                                                                                                                                                |\n",
      "|45 |xing|4500|[45, 4500]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500]]                                                                                                                                                                                    |\n",
      "|47 |xing|4700|[47, 4700]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700]]                                                                                                                                                                        |\n",
      "|53 |xing|5300|[53, 5300]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300]]                                                                                                                                                            |\n",
      "|55 |xing|5500|[55, 5500]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500]]                                                                                                                                                |\n",
      "|68 |xing|6800|[68, 6800]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800]]                                                                                                                                    |\n",
      "|72 |xing|7200|[72, 7200]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200]]                                                                                                                        |\n",
      "|73 |xing|7300|[73, 7300]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300]]                                                                                                            |\n",
      "|75 |xing|7500|[75, 7500]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500]]                                                                                                |\n",
      "|76 |xing|7600|[76, 7600]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600]]                                                                                    |\n",
      "|78 |xing|7800|[78, 7800]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800]]                                                                        |\n",
      "|82 |xing|8200|[82, 8200]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200]]                                                            |\n",
      "|85 |xing|8500|[85, 8500]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500]]                                                |\n",
      "|88 |xing|8800|[88, 8800]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800]]                                    |\n",
      "|92 |xing|9200|[92, 9200]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200]]                        |\n",
      "|94 |xing|9400|[94, 9400]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200], [94, 9400]]            |\n",
      "|99 |xing|9900|[99, 9900]|[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200], [94, 9400], [99, 9900]]|\n",
      "|1  |yun |100 |[1, 100]  |[[1, 100]]                                                                                                                                                                                                                                                                                                |\n",
      "|3  |yun |300 |[3, 300]  |[[1, 100], [3, 300]]                                                                                                                                                                                                                                                                                      |\n",
      "|4  |yun |400 |[4, 400]  |[[1, 100], [3, 300], [4, 400]]                                                                                                                                                                                                                                                                            |\n",
      "|9  |yun |900 |[9, 900]  |[[1, 100], [3, 300], [4, 400], [9, 900]]                                                                                                                                                                                                                                                                  |\n",
      "|12 |yun |1200|[12, 1200]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200]]                                                                                                                                                                                                                                                      |\n",
      "|16 |yun |1600|[16, 1600]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600]]                                                                                                                                                                                                                                          |\n",
      "|20 |yun |2000|[20, 2000]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000]]                                                                                                                                                                                                                              |\n",
      "|23 |yun |2300|[23, 2300]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300]]                                                                                                                                                                                                                  |\n",
      "|25 |yun |2500|[25, 2500]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500]]                                                                                                                                                                                                      |\n",
      "|28 |yun |2800|[28, 2800]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800]]                                                                                                                                                                                          |\n",
      "|34 |yun |3400|[34, 3400]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400]]                                                                                                                                                                              |\n",
      "|35 |yun |3500|[35, 3500]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500]]                                                                                                                                                                  |\n",
      "|46 |yun |4600|[46, 4600]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600]]                                                                                                                                                      |\n",
      "|50 |yun |5000|[50, 5000]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000]]                                                                                                                                          |\n",
      "|52 |yun |5200|[52, 5200]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200]]                                                                                                                              |\n",
      "|62 |yun |6200|[62, 6200]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200]]                                                                                                                  |\n",
      "|64 |yun |6400|[64, 6400]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400]]                                                                                                      |\n",
      "|65 |yun |6500|[65, 6500]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500]]                                                                                          |\n",
      "|67 |yun |6700|[67, 6700]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700]]                                                                              |\n",
      "|74 |yun |7400|[74, 7400]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400]]                                                                  |\n",
      "|84 |yun |8400|[84, 8400]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400]]                                                      |\n",
      "|89 |yun |8900|[89, 8900]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900]]                                          |\n",
      "|90 |yun |9000|[90, 9000]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000]]                              |\n",
      "|93 |yun |9300|[93, 9300]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000], [93, 9300]]                  |\n",
      "|97 |yun |9700|[97, 9700]|[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000], [93, 9300], [97, 9700]]      |\n",
      "+---+----+----+----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n",
      "root\n",
      " |-- t: integer (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- val: integer (nullable = true)\n",
      " |-- t_val: array (nullable = false)\n",
      " |    |-- element: integer (containsNull = true)\n",
      " |-- seq_t_val: array (nullable = false)\n",
      " |    |-- element: array (containsNull = false)\n",
      " |    |    |-- element: integer (containsNull = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import Window\n",
    "df_ex1_temp3 = df_ex1_temp2.withColumn('seq_t_val', F.collect_list(F.col('t_val')) \\\n",
    "                        .over(Window.partitionBy(F.col('name')) \\\n",
    "                        .orderBy(F.col('t').asc())))\n",
    "df_ex1_temp3.show(100, False)\n",
    "df_ex1_temp3.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8c0e3b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import ArrayType\n",
    "@F.udf(returnType=ArrayType(IntegerType()))\n",
    "def fun(arr:list, session_len:int):\n",
    "    last_time = arr[-1][0]\n",
    "    start_time = last_time - session_len\n",
    "    res = [i[1] for i in arr if i[0]>=start_time]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "41d170e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+\n",
      "|t  |seq_t_val                                                                                                                                                                                                                                                                                                 |seq_fin                                               |\n",
      "+---+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+\n",
      "|0  |[[0, 0]]                                                                                                                                                                                                                                                                                                  |[0]                                                   |\n",
      "|2  |[[0, 0], [2, 200]]                                                                                                                                                                                                                                                                                        |[0, 200]                                              |\n",
      "|5  |[[0, 0], [2, 200], [5, 500]]                                                                                                                                                                                                                                                                              |[0, 200, 500]                                         |\n",
      "|6  |[[0, 0], [2, 200], [5, 500], [6, 600]]                                                                                                                                                                                                                                                                    |[0, 200, 500, 600]                                    |\n",
      "|18 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800]]                                                                                                                                                                                                                                                        |[0, 200, 500, 600, 1800]                              |\n",
      "|19 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900]]                                                                                                                                                                                                                                            |[0, 200, 500, 600, 1800, 1900]                        |\n",
      "|24 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400]]                                                                                                                                                                                                                                |[500, 600, 1800, 1900, 2400]                          |\n",
      "|26 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600]]                                                                                                                                                                                                                    |[600, 1800, 1900, 2400, 2600]                         |\n",
      "|27 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700]]                                                                                                                                                                                                        |[1800, 1900, 2400, 2600, 2700]                        |\n",
      "|30 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000]]                                                                                                                                                                                            |[1800, 1900, 2400, 2600, 2700, 3000]                  |\n",
      "|31 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100]]                                                                                                                                                                                |[1800, 1900, 2400, 2600, 2700, 3000, 3100]            |\n",
      "|32 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200]]                                                                                                                                                                    |[1800, 1900, 2400, 2600, 2700, 3000, 3100, 3200]      |\n",
      "|38 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800]]                                                                                                                                                        |[1800, 1900, 2400, 2600, 2700, 3000, 3100, 3200, 3800]|\n",
      "|42 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200]]                                                                                                                                            |[2400, 2600, 2700, 3000, 3100, 3200, 3800, 4200]      |\n",
      "|43 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300]]                                                                                                                                |[2400, 2600, 2700, 3000, 3100, 3200, 3800, 4200, 4300]|\n",
      "|49 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900]]                                                                                                                    |[3000, 3100, 3200, 3800, 4200, 4300, 4900]            |\n",
      "|51 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100]]                                                                                                        |[3100, 3200, 3800, 4200, 4300, 4900, 5100]            |\n",
      "|54 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400]]                                                                                            |[3800, 4200, 4300, 4900, 5100, 5400]                  |\n",
      "|63 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300]]                                                                                |[4300, 4900, 5100, 5400, 6300]                        |\n",
      "|69 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900]]                                                                    |[4900, 5100, 5400, 6300, 6900]                        |\n",
      "|77 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700]]                                                        |[6300, 6900, 7700]                                    |\n",
      "|83 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300]]                                            |[6300, 6900, 7700, 8300]                              |\n",
      "|86 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600]]                                |[6900, 7700, 8300, 8600]                              |\n",
      "|87 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600], [87, 8700]]                    |[6900, 7700, 8300, 8600, 8700]                        |\n",
      "|95 |[[0, 0], [2, 200], [5, 500], [6, 600], [18, 1800], [19, 1900], [24, 2400], [26, 2600], [27, 2700], [30, 3000], [31, 3100], [32, 3200], [38, 3800], [42, 4200], [43, 4300], [49, 4900], [51, 5100], [54, 5400], [63, 6300], [69, 6900], [77, 7700], [83, 8300], [86, 8600], [87, 8700], [95, 9500]]        |[7700, 8300, 8600, 8700, 9500]                        |\n",
      "|8  |[[8, 800]]                                                                                                                                                                                                                                                                                                |[800]                                                 |\n",
      "|10 |[[8, 800], [10, 1000]]                                                                                                                                                                                                                                                                                    |[800, 1000]                                           |\n",
      "|11 |[[8, 800], [10, 1000], [11, 1100]]                                                                                                                                                                                                                                                                        |[800, 1000, 1100]                                     |\n",
      "|13 |[[8, 800], [10, 1000], [11, 1100], [13, 1300]]                                                                                                                                                                                                                                                            |[800, 1000, 1100, 1300]                               |\n",
      "|14 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400]]                                                                                                                                                                                                                                                |[800, 1000, 1100, 1300, 1400]                         |\n",
      "|29 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900]]                                                                                                                                                                                                                                    |[1000, 1100, 1300, 1400, 2900]                        |\n",
      "|36 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600]]                                                                                                                                                                                                                        |[2900, 3600]                                          |\n",
      "|39 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900]]                                                                                                                                                                                                            |[2900, 3600, 3900]                                    |\n",
      "|44 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400]]                                                                                                                                                                                                |[2900, 3600, 3900, 4400]                              |\n",
      "|48 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800]]                                                                                                                                                                                    |[2900, 3600, 3900, 4400, 4800]                        |\n",
      "|56 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600]]                                                                                                                                                                        |[3600, 3900, 4400, 4800, 5600]                        |\n",
      "|57 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700]]                                                                                                                                                            |[3900, 4400, 4800, 5600, 5700]                        |\n",
      "|58 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800]]                                                                                                                                                |[3900, 4400, 4800, 5600, 5700, 5800]                  |\n",
      "|59 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900]]                                                                                                                                    |[3900, 4400, 4800, 5600, 5700, 5800, 5900]            |\n",
      "|60 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000]]                                                                                                                        |[4400, 4800, 5600, 5700, 5800, 5900, 6000]            |\n",
      "|61 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100]]                                                                                                            |[4400, 4800, 5600, 5700, 5800, 5900, 6000, 6100]      |\n",
      "|66 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600]]                                                                                                |[4800, 5600, 5700, 5800, 5900, 6000, 6100, 6600]      |\n",
      "|70 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000]]                                                                                    |[5600, 5700, 5800, 5900, 6000, 6100, 6600, 7000]      |\n",
      "|71 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100]]                                                                        |[5600, 5700, 5800, 5900, 6000, 6100, 6600, 7000, 7100]|\n",
      "|79 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900]]                                                            |[5900, 6000, 6100, 6600, 7000, 7100, 7900]            |\n",
      "|80 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000]]                                                |[6000, 6100, 6600, 7000, 7100, 7900, 8000]            |\n",
      "|81 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100]]                                    |[6100, 6600, 7000, 7100, 7900, 8000, 8100]            |\n",
      "|91 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100]]                        |[7100, 7900, 8000, 8100, 9100]                        |\n",
      "|96 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100], [96, 9600]]            |[7900, 8000, 8100, 9100, 9600]                        |\n",
      "|98 |[[8, 800], [10, 1000], [11, 1100], [13, 1300], [14, 1400], [29, 2900], [36, 3600], [39, 3900], [44, 4400], [48, 4800], [56, 5600], [57, 5700], [58, 5800], [59, 5900], [60, 6000], [61, 6100], [66, 6600], [70, 7000], [71, 7100], [79, 7900], [80, 8000], [81, 8100], [91, 9100], [96, 9600], [98, 9800]]|[7900, 8000, 8100, 9100, 9600, 9800]                  |\n",
      "|7  |[[7, 700]]                                                                                                                                                                                                                                                                                                |[700]                                                 |\n",
      "|15 |[[7, 700], [15, 1500]]                                                                                                                                                                                                                                                                                    |[700, 1500]                                           |\n",
      "|17 |[[7, 700], [15, 1500], [17, 1700]]                                                                                                                                                                                                                                                                        |[700, 1500, 1700]                                     |\n",
      "|21 |[[7, 700], [15, 1500], [17, 1700], [21, 2100]]                                                                                                                                                                                                                                                            |[700, 1500, 1700, 2100]                               |\n",
      "|22 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200]]                                                                                                                                                                                                                                                |[700, 1500, 1700, 2100, 2200]                         |\n",
      "|33 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300]]                                                                                                                                                                                                                                    |[1500, 1700, 2100, 2200, 3300]                        |\n",
      "|37 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700]]                                                                                                                                                                                                                        |[1700, 2100, 2200, 3300, 3700]                        |\n",
      "|40 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000]]                                                                                                                                                                                                            |[2100, 2200, 3300, 3700, 4000]                        |\n",
      "|41 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100]]                                                                                                                                                                                                |[2100, 2200, 3300, 3700, 4000, 4100]                  |\n",
      "|45 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500]]                                                                                                                                                                                    |[3300, 3700, 4000, 4100, 4500]                        |\n",
      "|47 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700]]                                                                                                                                                                        |[3300, 3700, 4000, 4100, 4500, 4700]                  |\n",
      "|53 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300]]                                                                                                                                                            |[3300, 3700, 4000, 4100, 4500, 4700, 5300]            |\n",
      "|55 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500]]                                                                                                                                                |[3700, 4000, 4100, 4500, 4700, 5300, 5500]            |\n",
      "|68 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800]]                                                                                                                                    |[5300, 5500, 6800]                                    |\n",
      "|72 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200]]                                                                                                                        |[5300, 5500, 6800, 7200]                              |\n",
      "|73 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300]]                                                                                                            |[5300, 5500, 6800, 7200, 7300]                        |\n",
      "|75 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500]]                                                                                                |[5500, 6800, 7200, 7300, 7500]                        |\n",
      "|76 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600]]                                                                                    |[6800, 7200, 7300, 7500, 7600]                        |\n",
      "|78 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800]]                                                                        |[6800, 7200, 7300, 7500, 7600, 7800]                  |\n",
      "|82 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200]]                                                            |[6800, 7200, 7300, 7500, 7600, 7800, 8200]            |\n",
      "|85 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500]]                                                |[6800, 7200, 7300, 7500, 7600, 7800, 8200, 8500]      |\n",
      "|88 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800]]                                    |[6800, 7200, 7300, 7500, 7600, 7800, 8200, 8500, 8800]|\n",
      "|92 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200]]                        |[7200, 7300, 7500, 7600, 7800, 8200, 8500, 8800, 9200]|\n",
      "|94 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200], [94, 9400]]            |[7500, 7600, 7800, 8200, 8500, 8800, 9200, 9400]      |\n",
      "|99 |[[7, 700], [15, 1500], [17, 1700], [21, 2100], [22, 2200], [33, 3300], [37, 3700], [40, 4000], [41, 4100], [45, 4500], [47, 4700], [53, 5300], [55, 5500], [68, 6800], [72, 7200], [73, 7300], [75, 7500], [76, 7600], [78, 7800], [82, 8200], [85, 8500], [88, 8800], [92, 9200], [94, 9400], [99, 9900]]|[8200, 8500, 8800, 9200, 9400, 9900]                  |\n",
      "|1  |[[1, 100]]                                                                                                                                                                                                                                                                                                |[100]                                                 |\n",
      "|3  |[[1, 100], [3, 300]]                                                                                                                                                                                                                                                                                      |[100, 300]                                            |\n",
      "|4  |[[1, 100], [3, 300], [4, 400]]                                                                                                                                                                                                                                                                            |[100, 300, 400]                                       |\n",
      "|9  |[[1, 100], [3, 300], [4, 400], [9, 900]]                                                                                                                                                                                                                                                                  |[100, 300, 400, 900]                                  |\n",
      "|12 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200]]                                                                                                                                                                                                                                                      |[100, 300, 400, 900, 1200]                            |\n",
      "|16 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600]]                                                                                                                                                                                                                                          |[100, 300, 400, 900, 1200, 1600]                      |\n",
      "|20 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000]]                                                                                                                                                                                                                              |[100, 300, 400, 900, 1200, 1600, 2000]                |\n",
      "|23 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300]]                                                                                                                                                                                                                  |[300, 400, 900, 1200, 1600, 2000, 2300]               |\n",
      "|25 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500]]                                                                                                                                                                                                      |[900, 1200, 1600, 2000, 2300, 2500]                   |\n",
      "|28 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800]]                                                                                                                                                                                          |[900, 1200, 1600, 2000, 2300, 2500, 2800]             |\n",
      "|34 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400]]                                                                                                                                                                              |[1600, 2000, 2300, 2500, 2800, 3400]                  |\n",
      "|35 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500]]                                                                                                                                                                  |[1600, 2000, 2300, 2500, 2800, 3400, 3500]            |\n",
      "|46 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600]]                                                                                                                                                      |[2800, 3400, 3500, 4600]                              |\n",
      "|50 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000]]                                                                                                                                          |[3400, 3500, 4600, 5000]                              |\n",
      "|52 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200]]                                                                                                                              |[3400, 3500, 4600, 5000, 5200]                        |\n",
      "|62 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200]]                                                                                                                  |[4600, 5000, 5200, 6200]                              |\n",
      "|64 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400]]                                                                                                      |[4600, 5000, 5200, 6200, 6400]                        |\n",
      "|65 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500]]                                                                                          |[4600, 5000, 5200, 6200, 6400, 6500]                  |\n",
      "|67 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700]]                                                                              |[5000, 5200, 6200, 6400, 6500, 6700]                  |\n",
      "|74 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400]]                                                                  |[6200, 6400, 6500, 6700, 7400]                        |\n",
      "|84 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400]]                                                      |[6400, 6500, 6700, 7400, 8400]                        |\n",
      "|89 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900]]                                          |[7400, 8400, 8900]                                    |\n",
      "|90 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000]]                              |[7400, 8400, 8900, 9000]                              |\n",
      "|93 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000], [93, 9300]]                  |[7400, 8400, 8900, 9000, 9300]                        |\n",
      "|97 |[[1, 100], [3, 300], [4, 400], [9, 900], [12, 1200], [16, 1600], [20, 2000], [23, 2300], [25, 2500], [28, 2800], [34, 3400], [35, 3500], [46, 4600], [50, 5000], [52, 5200], [62, 6200], [64, 6400], [65, 6500], [67, 6700], [74, 7400], [84, 8400], [89, 8900], [90, 9000], [93, 9300], [97, 9700]]      |[8400, 8900, 9000, 9300, 9700]                        |\n",
      "+---+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_t = df_ex1_temp3.select(F.col('t'), F.col('seq_t_val'), fun(F.col('seq_t_val'), F.lit(20)).alias('seq_fin'))\n",
    "df_t.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "9597d8b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+--------------------------+\n",
      "|words    |features                  |\n",
      "+---------+--------------------------+\n",
      "|[a, b, c]|(10,[5,7,8],[1.0,1.0,1.0])|\n",
      "+---------+--------------------------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SparseVector(10, {5: 1.0, 7: 1.0, 8: 1.0})"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = spark.createDataFrame([([\"a\", \"b\", \"c\"],)], [\"words\"])\n",
    "from pyspark.ml.feature import HashingTF, Tokenizer\n",
    "hashingTF = HashingTF(inputCol=\"words\", outputCol=\"features\")\n",
    "hashingTF.setNumFeatures(10)\n",
    "hashingTF.transform(df).show(1, False)\n",
    "hashingTF.transform(df).head().features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "ebe554b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------------+-----+\n",
      "| id|             text|label|\n",
      "+---+-----------------+-----+\n",
      "|  0|  a b c d e spark|  1.0|\n",
      "|  1|            a c f|  0.0|\n",
      "|  2|spark hello world|  1.0|\n",
      "|  3| hadoop mapreduce|  0.0|\n",
      "|  4|     I love spark|  1.0|\n",
      "|  5|         big data|  0.0|\n",
      "+---+-----------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dftrain = spark.createDataFrame([(0,\"a b c d e spark\",1.0),\n",
    "                (1,\"a c f\",0.0),\n",
    "                (2,\"spark hello world\",1.0),\n",
    "                (3,\"hadoop mapreduce\",0.0),\n",
    "                (4,\"I love spark\", 1.0),\n",
    "                (5,\"big data\",0.0)],[\"id\",\"text\",\"label\"])\n",
    "dftrain.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "4caba9f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------------+-----+----------------------+\n",
      "|id |text             |label|words                 |\n",
      "+---+-----------------+-----+----------------------+\n",
      "|0  |a b c d e spark  |1.0  |[a, b, c, d, e, spark]|\n",
      "|1  |a c f            |0.0  |[a, c, f]             |\n",
      "|2  |spark hello world|1.0  |[spark, hello, world] |\n",
      "|3  |hadoop mapreduce |0.0  |[hadoop, mapreduce]   |\n",
      "|4  |I love spark     |1.0  |[i, love, spark]      |\n",
      "|5  |big data         |0.0  |[big, data]           |\n",
      "+---+-----------------+-----+----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer().setInputCol(\"text\").setOutputCol(\"words\")\n",
    "tokenizer.transform(dftrain).show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "17fb251b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------------+-----+----------------------+---------------------------------------------------+\n",
      "|id |text             |label|words                 |features                                           |\n",
      "+---+-----------------+-----+----------------------+---------------------------------------------------+\n",
      "|0  |a b c d e spark  |1.0  |[a, b, c, d, e, spark]|(100,[50,65,67,68,86,90],[1.0,1.0,1.0,1.0,1.0,1.0])|\n",
      "|1  |a c f            |0.0  |[a, c, f]             |(100,[67,68,94],[1.0,1.0,1.0])                     |\n",
      "|2  |spark hello world|1.0  |[spark, hello, world] |(100,[60,86,89],[1.0,1.0,1.0])                     |\n",
      "|3  |hadoop mapreduce |0.0  |[hadoop, mapreduce]   |(100,[50,85],[1.0,1.0])                            |\n",
      "|4  |I love spark     |1.0  |[i, love, spark]      |(100,[40,56,86],[1.0,1.0,1.0])                     |\n",
      "|5  |big data         |0.0  |[big, data]           |(100,[48,95],[1.0,1.0])                            |\n",
      "+---+-----------------+-----+----------------------+---------------------------------------------------+\n",
      "\n",
      "root\n",
      " |-- id: long (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      " |-- label: double (nullable = true)\n",
      " |-- words: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "hashingTF = HashingTF().setNumFeatures(100) \\\n",
    "   .setInputCol(tokenizer.getOutputCol()) \\\n",
    "   .setOutputCol(\"features\")\n",
    "hashingTF.transform(tokenizer.transform(dftrain)).show(100, False)\n",
    "hashingTF.transform(tokenizer.transform(dftrain)).printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "580b4511",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__UDT__',\n",
       " '__class__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__getitem__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__le__',\n",
       " '__len__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__',\n",
       " '__weakref__',\n",
       " 'dot',\n",
       " 'indices',\n",
       " 'norm',\n",
       " 'numNonzeros',\n",
       " 'size',\n",
       " 'squared_distance',\n",
       " 'toArray',\n",
       " 'values']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = hashingTF.transform(tokenizer.transform(dftrain)).collect()[0].asDict()['features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "d33767cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.,\n",
       "       1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.toArray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "6a94af00",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import Word2Vec, BucketedRandomProjectionLSH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "a67ce1e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Word2VecModel: uid=Word2Vec_d0224d21155d, numWords=16, vectorSize=3"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_document = spark.createDataFrame([\n",
    "    (\"Hi I heard about Spark\".split(\" \"), ),\n",
    "    (\"I wish Java could use case classes\".split(\" \"), ),\n",
    "    (\"Logistic regression models are neat\".split(\" \"), )\n",
    "], [\"text\"])\n",
    "\n",
    "word2Vec = Word2Vec(vectorSize=3, minCount=0, inputCol=\"text\", outputCol=\"result\")\n",
    "model = word2Vec.fit(df_document)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b60e29f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------------------+----------------------------------------------------------------+\n",
      "|text                                      |result                                                          |\n",
      "+------------------------------------------+----------------------------------------------------------------+\n",
      "|[Hi, I, heard, about, Spark]              |[-0.039981891959905626,0.01746610961854458,0.04743932448327542] |\n",
      "|[I, wish, Java, could, use, case, classes]|[-0.00982347849224295,0.020626994648150033,0.01723832743508475] |\n",
      "|[Logistic, regression, models, are, neat] |[0.039346391893923284,-0.012063178420066835,0.05037408620119095]|\n",
      "+------------------------------------------+----------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_vector = model.transform(df_document)\n",
    "df_vector.show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "8a94010f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----------------------------------------------------------------+\n",
      "|word      |vector                                                           |\n",
      "+----------+-----------------------------------------------------------------+\n",
      "|heard     |[0.08958848565816879,0.026097867637872696,0.14965184032917023]   |\n",
      "|are       |[0.14136365056037903,0.11820219457149506,0.13794642686843872]    |\n",
      "|neat      |[-0.1406438648700714,-0.15727408230304718,-0.0336344838142395]   |\n",
      "|classes   |[-0.14243242144584656,0.00829707458615303,-0.10411835461854935]  |\n",
      "|I         |[0.03133590519428253,0.06874048709869385,-0.06621349602937698]   |\n",
      "|regression|[0.07961297035217285,0.030162803828716278,0.10745978355407715]   |\n",
      "|Logistic  |[-0.014173711650073528,-0.08562734723091125,-0.07149039208889008]|\n",
      "|Spark     |[-0.15996260941028595,0.06433703005313873,-0.024921366944909096] |\n",
      "|could     |[0.02475137822329998,0.14349733293056488,-0.01949208974838257]   |\n",
      "|use       |[0.06559564173221588,0.07250787317752838,0.16070674359798431]    |\n",
      "|Hi        |[-0.03917152062058449,-0.05111256241798401,0.06395821273326874]  |\n",
      "|models    |[0.13057291507720947,0.03422053903341293,0.11158909648656845]    |\n",
      "|case      |[-0.07056943327188492,-0.14844214916229248,0.11728635430335999]  |\n",
      "|about     |[-0.12169972062110901,-0.020732274278998375,0.11472143232822418] |\n",
      "|Java      |[-0.023053064942359924,-0.09582533687353134,0.06421088427305222] |\n",
      "|wish      |[0.04560764506459236,0.09561368077993393,-0.031711749732494354]  |\n",
      "+----------+-----------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.getVectors().show(1000, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "7da85393",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.19990945979952812"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-0.03917152062058449 + 0.03133590519428253 + 0.08958848565816879 + (-0.12169972062110901) + (-0.15996260941028595)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "c7b245a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.039981891959905626"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-0.19990945979952812 / 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "5ea03579",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.ml.linalg.DenseVector"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.getVectors().collect()[0].asDict()['vector'].__class__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "4eeacf11",
   "metadata": {},
   "outputs": [],
   "source": [
    "brp = BucketedRandomProjectionLSH()\n",
    "brp.setInputCol('vector')\n",
    "brp.setOutputCol('lsh_brp')\n",
    "brp.setSeed(2022)\n",
    "brp.setBucketLength(10.0)\n",
    "brp_model = brp.fit(model.getVectors())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "e7176022",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----------------------------------------------------------------+--------+\n",
      "|word      |vector                                                           |lsh_brp |\n",
      "+----------+-----------------------------------------------------------------+--------+\n",
      "|heard     |[0.08958848565816879,0.026097867637872696,0.14965184032917023]   |[[0.0]] |\n",
      "|are       |[0.14136365056037903,0.11820219457149506,0.13794642686843872]    |[[-1.0]]|\n",
      "|neat      |[-0.1406438648700714,-0.15727408230304718,-0.0336344838142395]   |[[0.0]] |\n",
      "|classes   |[-0.14243242144584656,0.00829707458615303,-0.10411835461854935]  |[[-1.0]]|\n",
      "|I         |[0.03133590519428253,0.06874048709869385,-0.06621349602937698]   |[[-1.0]]|\n",
      "|regression|[0.07961297035217285,0.030162803828716278,0.10745978355407715]   |[[0.0]] |\n",
      "|Logistic  |[-0.014173711650073528,-0.08562734723091125,-0.07149039208889008]|[[0.0]] |\n",
      "|Spark     |[-0.15996260941028595,0.06433703005313873,-0.024921366944909096] |[[-1.0]]|\n",
      "|could     |[0.02475137822329998,0.14349733293056488,-0.01949208974838257]   |[[-1.0]]|\n",
      "|use       |[0.06559564173221588,0.07250787317752838,0.16070674359798431]    |[[-1.0]]|\n",
      "|Hi        |[-0.03917152062058449,-0.05111256241798401,0.06395821273326874]  |[[0.0]] |\n",
      "|models    |[0.13057291507720947,0.03422053903341293,0.11158909648656845]    |[[0.0]] |\n",
      "|case      |[-0.07056943327188492,-0.14844214916229248,0.11728635430335999]  |[[0.0]] |\n",
      "|about     |[-0.12169972062110901,-0.020732274278998375,0.11472143232822418] |[[0.0]] |\n",
      "|Java      |[-0.023053064942359924,-0.09582533687353134,0.06421088427305222] |[[0.0]] |\n",
      "|wish      |[0.04560764506459236,0.09561368077993393,-0.031711749732494354]  |[[-1.0]]|\n",
      "+----------+-----------------------------------------------------------------+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "brp_model.transform(model.getVectors()).show(100, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "8a52a11a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.column.Column"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyspark.ml.feature import OneHotEncoder\n",
    "df = spark.createDataFrame([(0.0,), (1.0,), (2.0,)], [\"input\"])\n",
    "ohe = OneHotEncoder(dropLast=False)\n",
    "ohe.setInputCols([\"input\"])\n",
    "ohe.setOutputCols([\"output\"])\n",
    "model = ohe.fit(df)\n",
    "model.setOutputCols([\"output\"])\n",
    "a = model.transform(df)\n",
    "# a = a.withColumn('one_hot', F.col('output').)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e62789ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "@F.udf(returnType=pyspark.ml.linalg.DenseVector())\n",
    "def fun(arr:list, session_len:int):\n",
    "    last_time = arr[-1][0]\n",
    "    start_time = last_time - session_len\n",
    "    res = [i[1] for i in arr if i[0]>=start_time]\n",
    "    return res"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
